import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import os

from numpy import double

from view import (pingLunView, reDuView, doubleView,
                  priceView, pingFengView, maxReDuView, maxPingFengView)


def getUrls():
	urls = []  # 列表
	baseUrl = "https://you.ctrip.com/sight/guizhou100064/s0-p"
	for i in range(1, 51):
		url = "{}{}.html".format(baseUrl, i)
		urls.append(url)  # 把生成好的 url 添加到列表 urls 当中方便返回
	return urls


headers = {
	"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
}


def gethtmltext(url):
	# 发送请求
	try:
		r = requests.get(url, headers=headers)
		r.raise_for_status()
		return r.text
	except:
		return ""


def ParsePage(html):
	if html == '':
		return
	soup = BeautifulSoup(html, features="html.parser")
	# 标题,名称,区域,户型,面积,朝向,装修,楼层,结构,关注,时间,总价,单价,标签
	info = soup.select("div ._39eNjW0iiiTo-HUAn0ADWF")
	# 定义一个列表 用于存储一页的数据
	data = []
	# 格式打印
	for item in info:
		# 数据不全
		if len(item.contents) != 4:
			continue
		name = item.contents[0].contents[0].contents[0].text
		if name.find('会') != -1 or name.find('音乐') != -1:
			continue
		koubei = item.contents[1].contents[0].text
		if item.contents[2].text.find("暂无评分") != -1:
			redu = 0
			pingfeng = 0
			pinglun = 0
		else:
			redu = double(item.contents[2].contents[0].contents[1].text) * 10
			pingfeng = double(item.contents[2].contents[2].contents[1].text) * 10
			pinglun = item.contents[2].contents[2].contents[2].text.replace("条点评", "")
			if pinglun.find("万") != -1:
				pinglun = pinglun.replace("万", "")
				pinglun = double(pinglun) * 10000

		price = item.contents[3].contents[1].text
		if price.find("免费") != -1 or price == "":
			price = 0
		else:
			if len(item.contents[3].contents[1].contents[0]) == 1:
				price = item.contents[3].contents[1].contents[0].contents[0].contents[1].contents[2].text
			else:
				price = item.contents[3].contents[1].contents[0].contents[1].contents[1].contents[2].text
		data.append([name, koubei, int(redu), int(pingfeng), int(pinglun), int(double(price))])
	return data


def getData():
	if os.path.exists('data.csv'):
		os.remove('data.csv')

	urls = getUrls()  # 返回值是一个列表
	for i in range(len(urls)):
		# 用 html 保存获取到的网页的所有信息
		html = gethtmltext(urls[i])
		# 解析每一页的数据
		data = ParsePage(html)
		# # 把数据写入文件
		df = pd.DataFrame(data)  # DataFrame 数据帧 二维表格结构
		df.to_csv("data.csv", mode="a", index=False, header=False, encoding="utf-8_sig")  # 追加
		print("第{}页写入完毕！".format(i + 1))
		time.sleep(1)  # 延时等待1秒


def draw():
	if os.path.exists('data.csv'):
		# 评分0-50 热度0-100
		columns = ["名称", "口碑", "热度", "评分", "评论", "价格"]
		df = pd.read_csv("data.csv")
		df = df.dropna()
		df = df.drop_duplicates()
		df.columns = columns
		pingLunView(df)
		pingFengView(df)
		priceView(df)
		reDuView(df)
		doubleView(df)
		maxReDuView(df)
		maxPingFengView(df)
		# 保存数据到hive数据库
		table_columns = ["name", "koubei", "heat", "score", "comment", "price"]
		df.columns = table_columns


if __name__ == "__main__":
	getData()
	draw()
